{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyalink.alink import *\n",
    "useLocalEnv(1)\n",
    "\n",
    "from utils import *\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "pd.set_option('display.max_colwidth', 1000)\n",
    "pd.set_option('display.max_rows', 100)\n",
    "\n",
    "DATA_DIR = ROOT_DIR + \"movielens\" + os.sep + \"ml-100k\" + os.sep\n",
    "\n",
    "RATING_FILE = \"u.data\";\n",
    "USER_FILE = \"u.user\";\n",
    "ITEM_FILE = \"u.item\";\n",
    "RATING_TRAIN_FILE = \"ua.base\";\n",
    "RATING_TEST_FILE = \"ua.test\";\n",
    "\n",
    "USER_COL = \"user_id\";\n",
    "ITEM_COL = \"item_id\";\n",
    "RATING_COL = \"rating\";\n",
    "RECOMM_COL = \"recomm\";\n",
    "\n",
    "ALS_MODEL_FILE = \"als_model.ak\";\n",
    "ITEMCF_MODEL_FILE = \"itemcf_model.ak\";\n",
    "USERCF_MODEL_FILE = \"usercf_model.ak\";\n",
    "\n",
    "RATING_SCHEMA_STRING = \"user_id long, item_id long, rating float, ts long\";\n",
    "\n",
    "USER_SCHEMA_STRING\\\n",
    "    = \"user_id long, age int, gender string, occupation string, zip_code string\";\n",
    "\n",
    "ITEM_SCHEMA_STRING = \"item_id long, title string, \"\\\n",
    "    + \"release_date string, video_release_date string, imdb_url string, \"\\\n",
    "    + \"unknown int, action int, adventure int, animation int, \"\\\n",
    "    + \"children int, comedy int, crime int, documentary int, drama int, \"\\\n",
    "    + \"fantasy int, film_noir int, horror int, musical int, mystery int, \"\\\n",
    "    + \"romance int, sci_fi int, thriller int, war int, western int\";\n",
    "\n",
    "\n",
    "def getSourceRatings() :\n",
    "    return TsvSourceBatchOp()\\\n",
    "            .setFilePath(DATA_DIR + RATING_FILE)\\\n",
    "            .setSchemaStr(RATING_SCHEMA_STRING);\n",
    "\n",
    "\n",
    "def getStreamSourceRatings() :\n",
    "    return TsvSourceStreamOp()\\\n",
    "            .setFilePath(DATA_DIR + RATING_FILE)\\\n",
    "            .setSchemaStr(RATING_SCHEMA_STRING);\n",
    "\n",
    "\n",
    "def getSourceUsers() :\n",
    "    return CsvSourceBatchOp()\\\n",
    "            .setFieldDelimiter(\"|\")\\\n",
    "            .setFilePath(DATA_DIR + USER_FILE)\\\n",
    "            .setSchemaStr(USER_SCHEMA_STRING);\n",
    "\n",
    "\n",
    "def getSourceItems() :\n",
    "    return CsvSourceBatchOp()\\\n",
    "            .setFieldDelimiter(\"|\")\\\n",
    "            .setFilePath(DATA_DIR + ITEM_FILE)\\\n",
    "            .setSchemaStr(ITEM_SCHEMA_STRING);\n",
    "\n",
    "\n",
    "def getStreamSourceItems() :\n",
    "    return CsvSourceStreamOp()\\\n",
    "            .setFieldDelimiter(\"|\")\\\n",
    "            .setFilePath(DATA_DIR + ITEM_FILE)\\\n",
    "            .setSchemaStr(ITEM_SCHEMA_STRING);\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_4\n",
    "\n",
    "train_set = TsvSourceBatchOp()\\\n",
    "    .setFilePath(DATA_DIR + RATING_TRAIN_FILE)\\\n",
    "    .setSchemaStr(RATING_SCHEMA_STRING);\n",
    "\n",
    "test_set = TsvSourceBatchOp()\\\n",
    "    .setFilePath(DATA_DIR + RATING_TEST_FILE)\\\n",
    "    .setSchemaStr(RATING_SCHEMA_STRING);\n",
    "\n",
    "if not(os.path.exists(DATA_DIR + ALS_MODEL_FILE)) :\n",
    "    train_set\\\n",
    "        .link(\n",
    "            AlsTrainBatchOp()\\\n",
    "                .setUserCol(USER_COL)\\\n",
    "                .setItemCol(ITEM_COL)\\\n",
    "                .setRateCol(RATING_COL)\\\n",
    "                .setLambda(0.1)\\\n",
    "                .setRank(10)\\\n",
    "                .setNumIter(10)\n",
    "        )\\\n",
    "        .link(\n",
    "            AkSinkBatchOp().setFilePath(DATA_DIR + ALS_MODEL_FILE)\n",
    "        );\n",
    "    BatchOperator.execute();\n",
    "\n",
    "\n",
    "PipelineModel(\n",
    "    AlsRateRecommender()\\\n",
    "        .setUserCol(USER_COL)\\\n",
    "        .setItemCol(ITEM_COL)\\\n",
    "        .setRecommCol(RECOMM_COL)\\\n",
    "        .setModelData(AkSourceBatchOp().setFilePath(DATA_DIR + ALS_MODEL_FILE)),\n",
    "    Lookup()\\\n",
    "        .setSelectedCols([ITEM_COL])\\\n",
    "        .setOutputCols([\"item_name\"])\\\n",
    "        .setModelData(getSourceItems())\\\n",
    "        .setMapKeyCols([\"item_id\"])\\\n",
    "        .setMapValueCols([\"title\"])\n",
    "    )\\\n",
    "    .transform(test_set.filter(\"user_id=1\"))\\\n",
    "    .select(\"user_id, rating, recomm, item_name\")\\\n",
    "    .orderBy(\"rating, recomm\", 1000)\\\n",
    "    .lazyPrint(-1);\n",
    "BatchOperator.execute();\n",
    "\n",
    "AlsRateRecommender()\\\n",
    "    .setUserCol(USER_COL)\\\n",
    "    .setItemCol(ITEM_COL)\\\n",
    "    .setRecommCol(RECOMM_COL)\\\n",
    "    .setModelData(AkSourceBatchOp().setFilePath(DATA_DIR + ALS_MODEL_FILE))\\\n",
    "    .transform(test_set)\\\n",
    "    .link(\n",
    "        EvalRegressionBatchOp()\\\n",
    "            .setLabelCol(RATING_COL)\\\n",
    "            .setPredictionCol(RECOMM_COL)\\\n",
    "            .lazyPrintMetrics()\n",
    "    );\n",
    "BatchOperator.execute();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_5\n",
    "\n",
    "if not(os.path.exists(DATA_DIR + ITEMCF_MODEL_FILE)) :\n",
    "    getSourceRatings()\\\n",
    "        .link(\n",
    "            ItemCfTrainBatchOp()\\\n",
    "                .setUserCol(USER_COL)\\\n",
    "                .setItemCol(ITEM_COL)\\\n",
    "                .setRateCol(RATING_COL)\n",
    "        )\\\n",
    "        .link(\n",
    "            AkSinkBatchOp().setFilePath(DATA_DIR + ITEMCF_MODEL_FILE)\n",
    "        );\n",
    "    BatchOperator.execute();\n",
    "\n",
    "test_data = BatchOperator.fromDataframe(pd.DataFrame([ [1] ]) , schemaStr='user_id long')\n",
    "\n",
    "ItemCfItemsPerUserRecommender()\\\n",
    "    .setUserCol(USER_COL)\\\n",
    "    .setRecommCol(RECOMM_COL)\\\n",
    "    .setModelData(AkSourceBatchOp().setFilePath(DATA_DIR + ITEMCF_MODEL_FILE))\\\n",
    "    .transform(test_data)\\\n",
    "    .print();\n",
    "\n",
    "recomm_predictor = ItemCfItemsPerUserRecommender()\\\n",
    "    .setUserCol(USER_COL)\\\n",
    "    .setRecommCol(RECOMM_COL)\\\n",
    "    .setK(20)\\\n",
    "    .setModelData(\n",
    "        AkSourceBatchOp().setFilePath(DATA_DIR + ITEMCF_MODEL_FILE)\n",
    "    )\\\n",
    "    .collectLocalPredictor(\"user_id long\");\n",
    "\n",
    "print(recomm_predictor.getOutputColNames());\n",
    "\n",
    "kv_predictor = Lookup()\\\n",
    "    .setSelectedCols([ITEM_COL])\\\n",
    "    .setOutputCols([\"item_name\"])\\\n",
    "    .setModelData(getSourceItems())\\\n",
    "    .setMapKeyCols([\"item_id\"])\\\n",
    "    .setMapValueCols([\"title\"])\\\n",
    "    .collectLocalPredictor(\"item_id long\");\n",
    "\n",
    "print(kv_predictor.getOutputColNames());\n",
    "\n",
    "recommResultStr = recomm_predictor.map([1])[1];\n",
    "\n",
    "print(recommResultStr);\n",
    "\n",
    "\n",
    "import json\n",
    "\n",
    "for id in eval(json.loads(recommResultStr).get('item_id')):\n",
    "    print(kv_predictor.map([id]));\n",
    "\n",
    "    \n",
    "Lookup()\\\n",
    "    .setSelectedCols([ITEM_COL])\\\n",
    "    .setOutputCols([\"item_name\"])\\\n",
    "    .setModelData(getSourceItems())\\\n",
    "    .setMapKeyCols([\"item_id\"])\\\n",
    "    .setMapValueCols([\"title\"])\\\n",
    "    .transform(getSourceRatings().filter(\"user_id=1 AND rating>4\"))\\\n",
    "    .select(\"item_name\")\\\n",
    "    .orderBy(\"item_name\", 1000)\\\n",
    "    .print()\n",
    "\n",
    "recomm_predictor_2 = ItemCfItemsPerUserRecommender()\\\n",
    "    .setUserCol(USER_COL)\\\n",
    "    .setRecommCol(RECOMM_COL)\\\n",
    "    .setK(20)\\\n",
    "    .setExcludeKnown(True)\\\n",
    "    .setModelData(\n",
    "        AkSourceBatchOp().setFilePath(DATA_DIR + ITEMCF_MODEL_FILE)\n",
    "    )\\\n",
    "    .collectLocalPredictor(\"user_id long\");\n",
    "\n",
    "recommResultStr = recomm_predictor_2.map([1])[1];\n",
    "\n",
    "print(recommResultStr);\n",
    "\n",
    "for id in eval(json.loads(recommResultStr).get('item_id')):\n",
    "    print(kv_predictor.map([id]));\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_6\n",
    "\n",
    "test_data = BatchOperator\\\n",
    "    .fromDataframe(\n",
    "        pd.DataFrame([ [50] ]), \n",
    "        schemaStr=ITEM_COL + ' long'\n",
    "    );\n",
    "\n",
    "ItemCfSimilarItemsRecommender()\\\n",
    "    .setItemCol(ITEM_COL)\\\n",
    "    .setRecommCol(RECOMM_COL)\\\n",
    "    .setModelData(AkSourceBatchOp().setFilePath(DATA_DIR + ITEMCF_MODEL_FILE))\\\n",
    "    .transform(test_data)\\\n",
    "    .print();\n",
    "\n",
    "recomm_predictor = ItemCfSimilarItemsRecommender()\\\n",
    ".setItemCol(ITEM_COL)\\\n",
    ".setRecommCol(RECOMM_COL)\\\n",
    ".setK(10)\\\n",
    ".setModelData(\n",
    "    AkSourceBatchOp().setFilePath(DATA_DIR + ITEMCF_MODEL_FILE)\n",
    ")\\\n",
    ".collectLocalPredictor(\"item_id long\");\n",
    "\n",
    "kv_predictor = Lookup()\\\n",
    ".setSelectedCols([ITEM_COL])\\\n",
    ".setOutputCols([\"item_name\"])\\\n",
    ".setModelData(getSourceItems())\\\n",
    ".setMapKeyCols([\"item_id\"])\\\n",
    ".setMapValueCols([\"title\"])\\\n",
    ".collectLocalPredictor(\"item_id long\");\n",
    "\n",
    "recommResultStr = recomm_predictor.map([50])[1];\n",
    "\n",
    "for id in eval(json.loads(recommResultStr).get('item_id')):\n",
    "    print(kv_predictor.map([id]));\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_7\n",
    "\n",
    "if not(os.path.exists(DATA_DIR + USERCF_MODEL_FILE)) :\n",
    "    getSourceRatings()\\\n",
    "    .link(\n",
    "        UserCfTrainBatchOp()\\\n",
    "            .setUserCol(USER_COL)\\\n",
    "            .setItemCol(ITEM_COL)\\\n",
    "            .setRateCol(RATING_COL)\n",
    "    )\\\n",
    "    .link(\n",
    "        AkSinkBatchOp().setFilePath(DATA_DIR + USERCF_MODEL_FILE)\n",
    "    );\n",
    "    BatchOperator.execute();\n",
    "\n",
    "\n",
    "test_data = BatchOperator\\\n",
    "    .fromDataframe(\n",
    "        pd.DataFrame([ [50] ]), \n",
    "        schemaStr=ITEM_COL + ' long'\n",
    "    )\n",
    "\n",
    "UserCfUsersPerItemRecommender()\\\n",
    "    .setItemCol(ITEM_COL)\\\n",
    "    .setRecommCol(RECOMM_COL)\\\n",
    "    .setModelData(AkSourceBatchOp().setFilePath(DATA_DIR + USERCF_MODEL_FILE))\\\n",
    "    .transform(test_data)\\\n",
    "    .print();\n",
    "\n",
    "getSourceRatings()\\\n",
    "    .filter(\"user_id IN (276,429,222,864,194,650,896,303,749,301) AND item_id=50\")\\\n",
    "    .print();\n",
    "\n",
    "UserCfUsersPerItemRecommender()\\\n",
    "    .setItemCol(ITEM_COL)\\\n",
    "    .setRecommCol(RECOMM_COL)\\\n",
    "    .setExcludeKnown(True)\\\n",
    "    .setModelData(AkSourceBatchOp().setFilePath(DATA_DIR + USERCF_MODEL_FILE))\\\n",
    "    .transform(test_data)\\\n",
    "    .print();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_8\n",
    "\n",
    "test_data = BatchOperator\\\n",
    "    .fromDataframe(\n",
    "        pd.DataFrame([ [1] ]), \n",
    "        schemaStr=USER_COL + ' long'\n",
    "    );\n",
    "\n",
    "UserCfSimilarUsersRecommender()\\\n",
    "    .setUserCol(USER_COL)\\\n",
    "    .setRecommCol(RECOMM_COL)\\\n",
    "    .setModelData(AkSourceBatchOp().setFilePath(DATA_DIR + USERCF_MODEL_FILE))\\\n",
    "    .transform(test_data)\\\n",
    "    .print();\n",
    "\n",
    "getSourceUsers()\\\n",
    "    .filter(\"user_id IN (1, 916,864,268,92,435,457,738,429,303,276)\")\\\n",
    "    .print();\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
